
static void balance_push(struct rq *rq);

/*
 * balance_push_callback is a right abuse of the callback interface and plays
 * by significantly different rules.
 *
 * Where the normal balance_callback's purpose is to be ran in the same context
 * that queued it (only later, when it's safe to drop rq->lock again),
 * balance_push_callback is specifically targeted at __schedule().
 *
 * This abuse is tolerated because it places all the unlikely/odd cases behind
 * a single test, namely: rq->balance_callback == NULL.
 */
struct balance_callback balance_push_callback = {
    .next = NULL,
    .func = balance_push,
};

#ifdef CONFIG_HOTPLUG_CPU

#else
static inline void balance_push(struct rq *rq)
{
}
#endif

static struct balance_callback *__splice_balance_callbacks(struct rq *rq, bool split)
{
    struct balance_callback *head = rq->balance_callback;

    if (likely(!head))
        return NULL;

    lockdep_assert_rq_held(rq);
    /*
     * Must not take balance_push_callback off the list when
     * splice_balance_callbacks() and balance_callbacks() are not
     * in the same rq->lock section.
     *
     * In that case it would be possible for __schedule() to interleave
     * and observe the list empty.
     */
    if (split && head == &balance_push_callback)
        head = NULL;
    else
        rq->balance_callback = NULL;

    return head;
}

static void do_balance_callbacks(struct rq *rq, struct balance_callback *head)
{
    void (*func)(struct rq *rq);
    struct balance_callback *next;

    lockdep_assert_rq_held(rq);

    while (head)
    {
        func = head->func;
        next = head->next;
        head->next = NULL;
        head = next;

        func(rq);
    }
}

static inline void __balance_callbacks(struct rq *rq)
{
    do_balance_callbacks(rq, __splice_balance_callbacks(rq, false));
}

static void balance_push_set(int cpu, bool on)
{
    struct rq *rq = cpu_rq(cpu);
    struct rq_flags rf;

    rq_lock_irqsave(rq, &rf);
    if (on)
    {
        WARN_ON_ONCE(rq->balance_callback);
        rq->balance_callback = &balance_push_callback;
    }
    else if (rq->balance_callback == &balance_push_callback)
    {
        rq->balance_callback = NULL;
    }
    rq_unlock_irqrestore(rq, &rf);
}
